{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pymongo\n",
    "from elasticsearch import Elasticsearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "db = pymongo.MongoClient('10.18.6.46',27001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "es = Elasticsearch(['10.18.6.102:9200'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc = db['db_parker']['jsl']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "item = doc.find_one()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'_id': ObjectId('5c249f29c4c05d4ba4bfa49d'),\n",
       " 'creator': 'greatbear',\n",
       " 'title': '各位研究技术指标有什么心得体会？',\n",
       " 'content': '各位研究技术指标多久了？研究这东西，能帮炒股赚钱么：）',\n",
       " 'resp_no': 11,\n",
       " 'createTime': '2018-12-14 08:38',\n",
       " 'crawlTime': '2019-03-16 09:54:43',\n",
       " 'url': 'https://www.jisilu.cn/question/297952',\n",
       " 'resp': [{'seeker24680_0': ['0', '可以说很多不明觉厉的词，适合用于忽悠别人，具体效果同风水先生。']},\n",
       "  {'joyfulli_1': ['0',\n",
       "    '首先要知道技术指标是怎么来的\\n对过去的数据进行统计分析，然后进行预测\\n说白了就是统计学\\n所以，学过统计学就知道技术分析怎么用了']},\n",
       "  {'老高0813_2': ['0', '基本没卵用']},\n",
       "  {'z383788052_3': ['0', '从来只看指标，只看k']},\n",
       "  {'花园小琴_4': ['0', '看图形，一般只看低买高卖，但事后后悔，赚少了']},\n",
       "  {'风险搬运工_5': ['0', '趋势（多和空）和震荡结合得做（同时做），不做单边、所谓轮动策略。\\n倾向于胜率低，赔率高的策略。']},\n",
       "  {'smag_6': ['0', '基本不看指标，静下心感受波动，在波动中下注。。。']},\n",
       "  {'海浪头头_7': ['0', '同意美棠子的看法']},\n",
       "  {'jsl0900_8': ['0', '从技术指标上看，我可以预测所有交易标的的后续走势，至于准确不准确，那不是我关心的问题']},\n",
       "  {'美棠子_9': ['4', '只是个辅助工具，基本面是核心，切不可颠倒主次。']},\n",
       "  {'蔓越橘_10': ['0', '用于T一下可以，其他就算了吧。']}],\n",
       " 'last_resp_date': '2018-12-14 10:14'}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "item"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "keys= item.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['_id', 'creator', 'title', 'content', 'resp_no', 'createTime', 'crawlTime', 'url', 'resp', 'last_resp_date'])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_db = db['db_parker']['jsl_note']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "AutoReconnect",
     "evalue": "10.18.6.46:27001: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTimeoutError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pymongo\\pool.py\u001b[0m in \u001b[0;36mreceive_message\u001b[1;34m(self, request_id)\u001b[0m\n\u001b[0;32m    609\u001b[0m             return receive_message(self.sock, request_id,\n\u001b[1;32m--> 610\u001b[1;33m                                    self.max_message_size)\n\u001b[0m\u001b[0;32m    611\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mBaseException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0merror\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pymongo\\network.py\u001b[0m in \u001b[0;36mreceive_message\u001b[1;34m(sock, request_id, max_message_size)\u001b[0m\n\u001b[0;32m    172\u001b[0m     length, _, response_to, op_code = _UNPACK_HEADER(\n\u001b[1;32m--> 173\u001b[1;33m         _receive_data_on_socket(sock, 16))\n\u001b[0m\u001b[0;32m    174\u001b[0m     \u001b[1;31m# No request_id for exhaust cursor \"getMore\".\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pymongo\\network.py\u001b[0m in \u001b[0;36m_receive_data_on_socket\u001b[1;34m(sock, length)\u001b[0m\n\u001b[0;32m    231\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 232\u001b[1;33m                 \u001b[0mchunk_length\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msock\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmv\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mbytes_read\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    233\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mIOError\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mOSError\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTimeoutError\u001b[0m: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mAutoReconnect\u001b[0m                             Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-37-2f3f08ea67f7>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0murl_set\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[1;32mfor\u001b[0m \u001b[0mitem\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mdoc\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m     \u001b[0mcreator\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'creator'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[0mtitle\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'title'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m     \u001b[0mcontent\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'content'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pymongo\\cursor.py\u001b[0m in \u001b[0;36mnext\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1187\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__empty\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1188\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1189\u001b[1;33m         \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__data\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_refresh\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1190\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__manipulate\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1191\u001b[0m                 \u001b[0m_db\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__collection\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdatabase\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pymongo\\cursor.py\u001b[0m in \u001b[0;36m_refresh\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1124\u001b[0m                                         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__collection\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdatabase\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclient\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1125\u001b[0m                                         self.__max_await_time_ms)\n\u001b[1;32m-> 1126\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__send_message\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mg\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1127\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1128\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__data\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pymongo\\cursor.py\u001b[0m in \u001b[0;36m__send_message\u001b[1;34m(self, operation)\u001b[0m\n\u001b[0;32m    929\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    930\u001b[0m                 response = client._send_message_with_response(\n\u001b[1;32m--> 931\u001b[1;33m                     operation, exhaust=self.__exhaust, address=self.__address)\n\u001b[0m\u001b[0;32m    932\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__address\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0maddress\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    933\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__exhaust\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pymongo\\mongo_client.py\u001b[0m in \u001b[0;36m_send_message_with_response\u001b[1;34m(self, operation, exhaust, address)\u001b[0m\n\u001b[0;32m   1143\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__all_credentials\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1144\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_event_listeners\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1145\u001b[1;33m             exhaust)\n\u001b[0m\u001b[0;32m   1146\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1147\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_reset_on_error\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mserver\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pymongo\\mongo_client.py\u001b[0m in \u001b[0;36m_reset_on_error\u001b[1;34m(self, server, func, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1154\u001b[0m         \"\"\"\n\u001b[0;32m   1155\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1156\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1157\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mNetworkTimeout\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1158\u001b[0m             \u001b[1;31m# The socket has been closed. Don't reset the server.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pymongo\\server.py\u001b[0m in \u001b[0;36msend_message_with_response\u001b[1;34m(self, operation, set_slave_okay, all_credentials, listeners, exhaust)\u001b[0m\n\u001b[0;32m    104\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    105\u001b[0m                 \u001b[0msock_info\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend_message\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax_doc_size\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 106\u001b[1;33m                 \u001b[0mreply\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msock_info\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreceive_message\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrequest_id\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    107\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    108\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mpublish\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pymongo\\pool.py\u001b[0m in \u001b[0;36mreceive_message\u001b[1;34m(self, request_id)\u001b[0m\n\u001b[0;32m    610\u001b[0m                                    self.max_message_size)\n\u001b[0;32m    611\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mBaseException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0merror\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 612\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_raise_connection_failure\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0merror\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    613\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    614\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_raise_if_not_writable\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0munacknowledged\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pymongo\\pool.py\u001b[0m in \u001b[0;36m_raise_connection_failure\u001b[1;34m(self, error)\u001b[0m\n\u001b[0;32m    741\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    742\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0merror\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msocket\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merror\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 743\u001b[1;33m             \u001b[0m_raise_connection_failure\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0maddress\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merror\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    744\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    745\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0merror\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pymongo\\pool.py\u001b[0m in \u001b[0;36m_raise_connection_failure\u001b[1;34m(address, error, msg_prefix)\u001b[0m\n\u001b[0;32m    281\u001b[0m         \u001b[1;32mraise\u001b[0m \u001b[0mNetworkTimeout\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    282\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 283\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mAutoReconnect\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    284\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    285\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAutoReconnect\u001b[0m: 10.18.6.46:27001: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。"
     ]
    }
   ],
   "source": [
    "url_set = set()\n",
    "for item in doc.find():\n",
    "    creator=item.get('creator')\n",
    "    title=item.get('title')\n",
    "    content=item.get('content')\n",
    "    resp_no=item.get('resp_no')\n",
    "    createTime=item.get('createTime')\n",
    "    \n",
    "    url=item.get('url')\n",
    "    if url in url_set:\n",
    "        continue\n",
    "    else:\n",
    "        url_set.add(url)\n",
    "    \n",
    "    if createTime is None:\n",
    "        save_db.insert_one({'url':url})\n",
    "        continue\n",
    "        \n",
    "    createTime=datetime.datetime.strptime(createTime,'%Y-%m-%d %H:%M').strftime('%Y-%m-%dT%H:%M:%S')\n",
    "\n",
    "    crawlTime=item.get('crawlTime')\n",
    "    crawlTime=datetime.datetime.strptime(crawlTime,'%Y-%m-%d %H:%M:%S').strftime('%Y-%m-%dT%H:%M:%S')\n",
    "\n",
    "    \n",
    "    resp_list=item.get('resp')\n",
    "    last_resp_date=item.get('last_resp_date')\n",
    "    last_resp_date=datetime.datetime.strptime(last_resp_date,'%Y-%m-%d %H:%M').strftime('%Y-%m-%dT%H:%M:%S')\n",
    "\n",
    "    _resp_list=[]\n",
    "    for sub_resp in resp_list:\n",
    "        resp_author = list(sub_resp.keys())[0]\n",
    "        _resp_author=''.join(resp_author.split('_')[:-1])\n",
    "        agree=sub_resp[resp_author][0]\n",
    "        resp_content=sub_resp[resp_author][1]\n",
    "        d={}\n",
    "        d['resp_agree']=int(agree)\n",
    "        d['resp_author']=_resp_author\n",
    "        d['resp_content']=resp_content\n",
    "\n",
    "        _resp_list.append(d)\n",
    "    # last_resp_date=item.get('last_resp_date')\n",
    "\n",
    "    body = {\n",
    "    'creator':creator,\n",
    "    'title':title,\n",
    "    'content':content,\n",
    "    'resp_no':int(resp_no),\n",
    "    'createTime':createTime,\n",
    "    'crawlTime':crawlTime,\n",
    "    'url':url,\n",
    "    'resp':_resp_list,\n",
    "    'last_resp_date':last_resp_date\n",
    "    }\n",
    "\n",
    "    es.index(index='jsl',doc_type='doc',body=body)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2019-03-16 09:54:43'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "crawlTime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'resp_agree': 0,\n",
       "  'resp_author': 'seeker24680',\n",
       "  'resp_content': '可以说很多不明觉厉的词，适合用于忽悠别人，具体效果同风水先生。'},\n",
       " {'resp_agree': 0,\n",
       "  'resp_author': 'joyfulli',\n",
       "  'resp_content': '首先要知道技术指标是怎么来的\\n对过去的数据进行统计分析，然后进行预测\\n说白了就是统计学\\n所以，学过统计学就知道技术分析怎么用了'},\n",
       " {'resp_agree': 0, 'resp_author': '老高0813', 'resp_content': '基本没卵用'},\n",
       " {'resp_agree': 0, 'resp_author': 'z383788052', 'resp_content': '从来只看指标，只看k'},\n",
       " {'resp_agree': 0,\n",
       "  'resp_author': '花园小琴',\n",
       "  'resp_content': '看图形，一般只看低买高卖，但事后后悔，赚少了'},\n",
       " {'resp_agree': 0,\n",
       "  'resp_author': '风险搬运工',\n",
       "  'resp_content': '趋势（多和空）和震荡结合得做（同时做），不做单边、所谓轮动策略。\\n倾向于胜率低，赔率高的策略。'},\n",
       " {'resp_agree': 0,\n",
       "  'resp_author': 'smag',\n",
       "  'resp_content': '基本不看指标，静下心感受波动，在波动中下注。。。'},\n",
       " {'resp_agree': 0, 'resp_author': '海浪头头', 'resp_content': '同意美棠子的看法'},\n",
       " {'resp_agree': 0,\n",
       "  'resp_author': 'jsl0900',\n",
       "  'resp_content': '从技术指标上看，我可以预测所有交易标的的后续走势，至于准确不准确，那不是我关心的问题'},\n",
       " {'resp_agree': 4,\n",
       "  'resp_author': '美棠子',\n",
       "  'resp_content': '只是个辅助工具，基本面是核心，切不可颠倒主次。'},\n",
       " {'resp_agree': 0, 'resp_author': '蔓越橘', 'resp_content': '用于T一下可以，其他就算了吧。'}]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "_resp_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for item in doc.find_one():\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "db"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
